/**
 * 理财-债券
 * http://roll.finance.sina.com.cn/finance/zq2/zsscdt/index.shtml
 */
var cheerio=require("cheerio");//引用抓取html结构的node模块
var mkdirp=require("mkdirp");  //引用创建文件夹的node模块
var iconv=require("iconv-lite"); //引用字符集node模块
var commen=require("../allocation/commen");//引用公共操作的自定义文件
var data=require("../allocation/sql");  //引用数据库操作的自定义文件
var async   = require("async");  //引用同步执行

var url       = "http://roll.finance.sina.com.cn/finance/zq2/zsscdt/index.shtml";
var chanel    = "zhaiquan";
var newsType  = 4;
var imgSite   = "";
var website   = "";
var newsLists = [];   //存放新闻列表
var logs=[];   //存放日志信息
var count=0;
//创建存放图片的文件夹及路径(20160901)
var imgPath               = commen.createDownFile().imgPath;

mkdirp.sync(imgPath);
logs.push({
	start_time:new Date().toLocaleString(),
	article_type_id:newsType,
	thread:chanel
});

//定义获取新闻列表
function getNewsList(url,callback)
{
	commen.load(url,function(err,body){
		if(err)
		{
			logs[0].location="列表请求";
			logs[0].message="错误："+err;
			logs[0].level=0;
			logs[0].end_time=new Date().toLocaleString();
			data.insertLog(logs[0],function(err,result){
				if(err) throw err;
				if(result>0)
				{
					console.log("列表请求出错了！！！!");
					process.exit(0);
				}
			});
			return err;
		}
		var  $  =cheerio.load(body);
		var articleList = [];
		//获取新闻列表
		$(".listBlk ul li").each(function (index) {
			if (index==21) {
				return false; 
			}
			if ($(this).find("a").attr("href")!=undefined) {
			 articleList.push(website+$(this).find("a").attr("href"));
			 }
		});
		callback(null, articleList);
		// console.log(articleList);
		// console.log(articleList.length);
		// process.exit(0);
	});
}
//定义获取新闻内容
function getNewsContent(url,callback)
{
	commen.load(url,function(err,body){
		if(err)
		{
			logs[0].location="内容请求";
			logs[0].message="错误："+err;
			logs[0].level=0;
			logs[0].end_time=new Date().toLocaleString();
			data.insertLog(logs[0],function(err,result){
				if(err) throw err;
				if(result>0)
				{
					console.log("内容请求出错了！！！");
					process.exit(0);
				}
			});
			return err;
		}
		// var html = iconv.decode(body, 'gb2312');
		var $           = cheerio.load(body, {decodeEntities: false});
		var news        = [];
		var picUrls     = []; //存放图片的数组
		var allContent  = "";  //存放排版好的ubb格式的内容
		// imgSite = url.substring(0,url.lastIndexOf("/")+1);
		if($("#artibody img").length>0)
		{
			for(var i=0;i<$("#artibody img").length;i++)
			{
				if($($("#artibody img")[i]).attr("src").indexOf("title")>0)
				{    
					$($("#artibody img")[i]).remove();
					continue; 
				}
					picUrls.push(imgSite+$($("#artibody img")[i]).attr("src"));
					$($("#artibody img")[i]).html("[text-align:center]"+imgSite+$($("#artibody img")[i]).attr("src")+"[/img]"+"[br/]");
					$($("#artibody img")[i]).removeAttr("title");
					$($("#artibody img")[i]).removeAttr("alt");
					$($("#artibody img")[i]).removeAttr("src");
			}
		}
		$("#artibody").find("script").remove();
		$("#artibody").find("style").remove();
		$("#artibody").find("blockquote").remove();
		$("#artibody div").last().remove();
		//去除内容html中的空格和样式
		if ($("#artibody").html()==null) {
			logs[0].location="解析标签";
			logs[0].message="去除标签错误：Cannot read property 'replace' of null";
			logs[0].level=0;
			logs[0].end_time=new Date().toLocaleString();
			data.insertLog(logs[0],function(err,result){
				if(err) throw err;
				if(result>0)
				{
					console.log("去除标签出错了！！！");
					process.exit(0);
				}
			});
			return err;

		}
		else
		{
		$("#artibody").html($("#artibody").html().replace(/<!--[\w\W\r\n]*?-->/gmi,'').trim()); 
		$("#artibody").html($("#artibody").html().replace(/<([a-zA-Z1-6]+)(\s*[^>]*)?>/g, "<$1>"));
		}
		//找到内容中所有的p标签
		for (var i = 0; i <$("#artibody p").length; i++) {
			if($($("#artibody p")[i]).find("img")=="")
			{
				$($("#artibody p")[i]).text("[&emsp][&emsp]" + $($("#artibody p")[i]).text().trim());
			}
			allContent += $($("#artibody p")[i]).text();
		}
		//替换内容中所有的&nbsp;$gt等标签为ubb格式  .replace(/(<\/?table.*?>)|(<\/?tbody.*?>)|(<\/?tr.*?>)|(<\/?td.*?>)/g, '')
		allContent = $("#artibody").html().trim().replace(/'/g, "").replace(/\s/g, "").replace(/>/g, "]").replace(/</g, "[").replace(/&rdquo;/g, "[&rdquo]").replace(/&ldquo;/g,"[&ldquo]").replace(/&nbsp;/g,"[&nbsp]").replace(/&amp;/g,"[&amp]").replace(/&gt;/g,"[&gt]").replace(/&lt;/g,"[&lt]");
		// console.log(allContent);
		//将插入数据库的值存入数组news中
		news.push({
			title:$("#artibodyTitle").text().trim(),
			source:'新浪财经',
			author:$(".article-editor")?$(".article-editor").text().trim():'wangluo',
			date:new Date().toLocaleString(),
			typeID:newsType,
			chanel:chanel,
			content:allContent,
			picUrls:picUrls
		});
		callback(null, news[0]);
		// console.log(news[0]);
	});
}

//调用方法，依次获取列表和内容，并存入数据库
async.series([
	// 获取新闻列表
	function (done) {
		getNewsList(url,function(err,list){
			newsLists = list;
			done(err);
		});
	},
	function (done) {
		async.eachSeries(newsLists, function (item, next) {
			getNewsContent(item, function (err,detail) {
				if (err) console.error(err.stack);
				data.insertData(detail,imgPath,function(err,result){
					if(err)
					{
						logs[0].location="入库";
						logs[0].message="错误："+err;
						logs[0].level=0;
						logs[0].end_time=new Date().toLocaleString();
						data.insertLog(logs[0],function(err,result){
							if(err) throw err;
							if(result>0)
							{
								console.log("入库出错了！！！");
								process.exit(0);
							}
						});
						return err;
					}
					else if(result>0)
                     {
                        console.log("采集成功");
            
                     }
				});
				next();
			});
		}, done);
	}
], function (err) {
	if (err) console.error(err.stack);
	logs[0].location="无";
	logs[0].message="正确";
	logs[0].level=1;
	logs[0].end_time=new Date().toLocaleString();
	data.insertLog(logs[0],function(err,result){
	 if(err) throw err;
	 if(result>0)
	  {
		 console.log("写入日志成功!");
		  console.log('采集完成');
		  process.exit(0);
	  }
	 });
});
